//
// Created by 李康 on 15/12/9.
//

#include "datapreprocess.h"
#include <fstream>
#include <vector>
#include <dirent.h>
#include <numeric>
#include <sys/stat.h>
#include <map>
#include <sstream>

using namespace std;

void QVP_data_preprocess(string s_dir, string d_dir, string dns_dir) {
    DIR *dp;
    struct dirent *dirp;
    struct stat filestat;
    string filename;
    string line;
    string domain_name;
    string dns_ip;
    string hour_str;
    int hour;
    unsigned long beginindex;
    unsigned long endindex;
    stringstream ss;
    vector<map<string, vector<long> > > record;
    vector<map<string, vector<long> > > total_dns;
    ifstream input;
    map<string, long>domain_total;
    if (mkdir(d_dir.c_str(), 0755)) {
        cout << "create " << d_dir << " failed!" << endl;
        return;
    }
    if (mkdir(dns_dir.c_str(), 0755)) {
        cout << "create " << d_dir << " failed!" << endl;
        return;
    }
    if ( !(dp = opendir(s_dir.c_str())) ) {
        cout << "open " << s_dir << "failed!" << endl;
        return;
    }
    while ((dirp = readdir(dp))) {
        filename = s_dir + dirp->d_name;
        //if the file is directory or invalid , skip it
        if (stat(filename.c_str(), &filestat)) {
            continue;
        }
        if (S_ISDIR(filestat.st_mode)) {
            continue;
        }
        cout << filename << endl;
        map<string, vector<long> > domain;
        map<string, vector<long> > dns_server;
        input.open(filename.c_str());
        if (!input) {
            cout << "open file failed!" << endl;
            return;
        }
        cout << "**********************************" << endl;
        //match domain name from DNS LOG file and dns server ip
        while (getline(input, line)) {
            //match domain name
            beginindex = line.find_first_of(" ");
            endindex = line.find("|", beginindex + 1);
            string::iterator b = line.begin();
            domain_name = string(b + beginindex + 1, b + endindex - 1);
            //match dns server ip
            beginindex = line.find("|", endindex + 1);
            dns_ip = string(b + endindex + 1, b + beginindex);
            //match time
            hour_str = string(line, 0, 2);
            ss << hour_str;
            ss >> hour;
            if (domain.find(domain_name) != domain.end()) {
                domain[domain_name][hour] += 1;
            }
            else {
                domain[domain_name].resize(24, 0);
                domain[domain_name][hour] = 1;
            }
            if (dns_server.find(dns_ip) != dns_server.end()) {
                dns_server[dns_ip][hour] += 1;
            }
            else {
                dns_server[dns_ip].resize(24, 0);
                dns_server[dns_ip][hour] = 1;
            }
            ss.clear();
        }
        cout << "**********************************" << endl;
        record.push_back(domain);
        total_dns.push_back(dns_server);
        input.close();
        input.clear();
    }
    /*************************************************
     * handle domain visited data
     *************************************************/
    for (vector<map<string, vector<long> > >::iterator iterator1 = record.begin(); iterator1 != record.end(); ++iterator1) {
        for (map<string, vector<long> >::iterator iterator2 = iterator1->begin(); iterator2 != iterator1->end(); ++iterator2) {
            domain_total[iterator2->first] += accumulate(iterator2->second.begin(), iterator2->second.end(), 0);
        }
    }
    vector<pair<string, long> >temp;
    for (map<string, long>::iterator iterator1 = domain_total.begin(); iterator1 != domain_total.end(); ++ iterator1) {
        temp.push_back(make_pair(iterator1->first, iterator1->second));
    }
    sort(temp.begin(), temp.end(), comByValue());
    vector<pair<string, long> >::iterator iter = temp.begin();
    int nums = 0;
    string outfilename;
    ofstream outfile;
    int id = 0;
    int counts = 0;
    vector<string> frequent_domain;
    while (true) {
        //the domain name visited count greater than 2100 in the past 7 days
        if (iter->second >= 2100) {
            frequent_domain.push_back(iter->first);
            id = 0;
            ++counts;
            outfilename = d_dir + iter->first;
            outfile.open(outfilename.c_str(), ofstream::app);
            for (vector<map<string, vector<long> > >::iterator iterator1 = record.begin(); iterator1 != record.end(); ++iterator1) {
                ++id;
                nums = 0;
                if ((*iterator1).find(iter->first) != (*iterator1).end()) {
                    for (vector<long>::iterator iterator2 = (*iterator1)[iter->first].begin(); iterator2 != (*iterator1)[iter->first].end(); ++iterator2) {
                        if (nums < 10) {
                            ss << "0" << nums;
                            ss >> hour_str;
                        }
                        else {
                            ss << nums;
                            ss >> hour_str;
                        }
                        ss.clear();
                        if (nums == 0) {
                            outfile << hour_str << "s ";
                        }
                        else {
                            if ((*iterator2) > 2 + *(iterator2 - 1)) {
                                outfile << hour_str << "u ";
                            }
                            else outfile << hour_str << ((*iterator2) < *(iterator2 - 1) - 2? "d " : "s ");
                        }
                        ++nums;
                    }
                    outfile << "\n";
                }
                else {
                    outfile << "00s 01s 02s 03s 04s 05s 06s 07s 08s 09s 10s 11s 12s 13s 14s 15s 16s 17s 18s 19s 20s 21s 22s 23s\n";
                }
            }
            outfile.close();
            outfile.clear();
            cout << iter->first << ":" << iter->second << endl;
            ++iter;
        }
        else {
            break;
        }
    }

    /*********************************************************
     * handle dns server visited data
     *********************************************************/
    for (vector<map<string, vector<long> > >::iterator iter = total_dns.begin(); iter != total_dns.end(); ++iter) {
        for (map<string, vector<long> >::iterator iter1 = iter->begin(); iter1 != iter->end(); ++iter1) {
            outfilename = dns_dir + iter1->first;
            outfile.open(outfilename.c_str(), ofstream::app);
            nums = 0;
            for (vector<long>::iterator iter2 = iter1->second.begin(); iter2 != iter1->second.end(); ++iter2) {
                if (nums < 10) {
                    ss << "0" << nums;
                    ss >> hour_str;
                }
                else {
                    ss << nums;
                    ss >> hour_str;
                }
                ss.clear();
                if (nums == 0) {
                    outfile << hour_str << "s ";
                }
                else {
                    if ((*iter2) > 200 + *(iter2 - 1)) {
                        outfile << hour_str << "u ";
                    }
                    else outfile << hour_str << ((*iter2) < *(iter2 - 1) - 200 ? "d " : "s ");
                }
                ++nums;
            }
            outfile << "\n";
            outfile.close();
            outfile.clear();
        }
    }

    /**********************************************************
     * Query Traffic Characteristic Analysis data process
     **********************************************************/

    long max = 0;
    long min = LONG_MAX;
    vector<pair<string, long> > maxvec;
    vector<pair<string, long> > minvec;
    cout << "counts:" << counts << endl;
    for (int i = 0; i < counts; ++i) {
        string name = frequent_domain[i];
        cout << "name:" << frequent_domain[i] << endl;
        for (vector< map<string, vector<long> > >::iterator iterator1 = record.begin(); iterator1 != record.end(); ++iterator1) {
            bool bb;
            bb = (*iterator1).find(frequent_domain[i]) != (*iterator1).end();
            if (bb) {
                maxvec.push_back(make_pair(frequent_domain[i], *max_element((*iterator1)[frequent_domain[i]].begin(), (*iterator1)[frequent_domain[i]].end())));
                minvec.push_back(make_pair(frequent_domain[i], *min_element((*iterator1)[frequent_domain[i]].begin(), (*iterator1)[frequent_domain[i]].end())));
            }
        }
    }
    sort(maxvec.begin(), maxvec.end(), comByValue());
    sort(minvec.begin(), minvec.end(), comByValue());
    //detect noise data and remove it from frequent_domain
    while (true) {
        vector<pair<string, long> >::iterator start = maxvec.begin();
        vector<pair<string, long> >::iterator finish = maxvec.end() - 1;
        if (start->second / finish->second > 1000) {
            maxvec.erase(start);
            vector<string>::iterator position = find(frequent_domain.begin(), frequent_domain.end(), start->first);
            if (position != frequent_domain.end()) {
                frequent_domain.erase(position);
                --counts;
            }
        }
        else if (finish->second <= 10){
            maxvec.erase(finish);
            vector<string>::iterator position = find(frequent_domain.begin(), frequent_domain.end(), finish->first);
            if (position != frequent_domain.end()) {
                frequent_domain.erase(position);
                --counts;
            }
        }
        else {
            break;
        }
    }
    for (vector<pair<string, long> >::iterator iterator1 = maxvec.begin(); iterator1 != maxvec.end(); ++iterator1) {
        cout << iterator1->first << " " << iterator1->second << endl;
    }
    cout << endl;
    cout << "counts:" <<counts << endl;
    max = maxvec.begin()->second;
    min = (minvec.end() - 1)->second;
    cout << "max:" << max << " min:" << min << endl;
    outfilename = "/Users/likang/ClionProjects/DNS_mining/QTCAdata";
    outfile.open(outfilename);
    //normalize frquent domain_name to 0..1
    for (int i = 0; i < counts; ++i) {
        outfile << frequent_domain[i] << " ";
        for (vector<map<string, vector<long> > >::iterator iterator1 = record.begin(); iterator1 != record.end(); ++iterator1) {
            bool bb;
            bb = (*iterator1).find(frequent_domain[i]) != (*iterator1).end();
            if (bb) {

                for (vector<long>::iterator iterator2 = (*iterator1)[frequent_domain[i]].begin(); iterator2 != (*iterator1)[frequent_domain[i]].end(); ++iterator2) {
                    outfile << (double)((*iterator2) - min) / (max - min) << " ";
                }
            }
            else {
                for (int j = 0; j < 24; ++j) {
                    outfile << "0.0 ";
                }
            }
        }
        outfile << "\n";
    }
    outfile.close();
    outfile.clear();
}
